package edu.uci.ics.crawler4j.examples.douban;

import au.com.bytecode.opencsv.CSVWriter;
import com.google.common.io.Files;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.fetcher.PageFetchResult;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;
import org.apache.http.Header;
import org.apache.http.HttpStatus;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.nio.charset.Charset;
import java.util.*;
import java.util.regex.Pattern;

/**
 * Created by liuyaowen on 2016/8/25.
 */
public class DoubanCrawler extends WebCrawler {
    private static final Pattern IMAGE_EXTENSIONS = Pattern.compile(".*\\.(bmp|gif|jpg|png)$");

    int index=0;
    List<DouBanContent> list=new ArrayList<>();



    @Override
    public boolean shouldVisit(Page referringPage, WebURL url)throws Exception {
        String href = url.getURL().toLowerCase();
        if (IMAGE_EXTENSIONS.matcher(href).matches()) {
            return false;
        }
        return href.startsWith("https://www.douban.com/group/topic/");
    }
    @Override
    public void visit(Page page) {
        logger.info("正在解析的URL：{"+index+"}"+page.getWebURL().getURL());
        HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
         Document doc = Jsoup.parse(htmlParseData.getHtml());
         Element contents = doc.select("div[id=content]").first();

        DouBanContent douBanContent=new DouBanContent();
        //处理标题
        String title=handleTitle(contents);
        //处理内容
        String content=handleContent(contents,page.getWebURL().getURL());
        //处理评论
        Set<String> reviews=handleReview(doc.select("ul[id=comments]").select("li"));

        douBanContent.setTitle(title);
        douBanContent.setContent(content);
        douBanContent.setReviews(reviews);
        index=index+1;
    try {
        logger.info(douBanContent.toString());
        //插入数据库
    }catch (Exception e){
        e.printStackTrace();
    }
        logger.debug("=============");
    }

    private Set<String> handleReview(Elements elements) {
        Set<String> hashSet=new HashSet<>();
        if (elements.size()>0){
            for (Element element:elements){
                String   review= element.select(".content p").text();
                hashSet.add(review);
            }
        }
        return hashSet;
    }

    private String handleContent(Element c,String sourceUrl) {
        StringBuffer sb=new StringBuffer();
        String content= c.select(".topic-doc p").toString();
        sb.append(content.replaceAll("<br>","\n").replaceAll("<p>","").replaceAll("</p>",""));
        Elements elements= c.select(".topic-doc .topic-figure img");
        if (elements.size()>0){
            sb.append("图片如下：\n");
            for (Element element:elements){
                String url=element.attr("src");
                try {
                    String imgName = url.split("/")[url.split("/").length-1];
                    String imgFileName="c:\\xxoo\\"+imgName;
                    //下载到本地
                    Downloader.download(url,imgName,"c:\\xxoo\\");
                    //上传到七牛
                    UploadUtils.upload(imgName,imgFileName);
                    //删除本地
                    File file=new File(imgFileName);
                    file.delete();
                    //写入文章内容
                    sb.append("[img]"+UploadUtils.QINIU_URL+imgName+"[/img]").append("\n");
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
        sb.append("\n来源："+sourceUrl);
        return sb.toString();
    }

    private String handleTitle(Element c) {
        String title= c.select("h1").first().text();
        if (title.indexOf("...")>-1){
            title= c.select(".tablecc").text().replaceAll("标题：","");
        }
        return title;
    }
}


